In [170]:
# Import libraries
import h5py
import numpy as np
import pandas as pd
import librosa
import librosa.display

import matplotlib.pyplot as plt
from matplotlib import image

import os

from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression, LassoCV
from sklearn.preprocessing import LabelBinarizer, StandardScaler, LabelEncoder
from sklearn.utils import class_weight
from sklearn.metrics import confusion_matrix, ConfusionMatrixDisplay, accuracy_score, precision_score, recall_score, f1_score, classification_report
from tensorflow.keras.preprocessing.image import load_img, img_to_array
from tensorflow.keras.preprocessing.sequence import pad_sequences
from tensorflow.keras.callbacks import EarlyStopping

from tensorflow.keras import models, layers
from tensorflow.keras.models import Sequential, load_model
from tensorflow.keras.layers import Dense, Dropout, Conv2D, MaxPooling2D, Flatten, Input
from tensorflow.keras.datasets import mnist, cifar10
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.applications.resnet import preprocess_input
from tensorflow.keras.applications import ResNet50
from tensorflow.keras.applications import imagenet_utils
In [3]:
# Evaluate hdf5 file. Check for key values
f = h5py.File('bird_spectrograms.hdf5', 'r')
for key in f.keys():
    print(key, f[key].shape)
amecro (128, 517, 66)
amerob (128, 517, 172)
bewwre (128, 517, 144)
bkcchi (128, 517, 45)
daejun (128, 517, 125)
houfin (128, 517, 84)
houspa (128, 517, 630)
norfli (128, 517, 37)
rewbla (128, 517, 187)
sonspa (128, 517, 263)
spotow (128, 517, 137)
whcspa (128, 517, 91)
In [5]:
# Example of Spectrogram for American Crow 
plt.figure(figsize=(10, 4))
librosa.display.specshow(f['amecro'][:,:,10], x_axis='time', y_axis='mel', cmap='gray_r')
plt.title('Mel Spectrogram: amecro clip 10')
plt.xlabel('Time')
plt.ylabel('Frequency (mel)')
plt.colorbar(format='%+2.0f dB')
plt.tight_layout()
plt.show()
No description has been provided for this image
In [7]:
# Add name identifier for species
bird_names = {
    "amecro": "American Crow",
    "amerob": "American Robin",
    "bewwre": "Bewick's Wren",
    "bkcchi": "Black-capped Chickadee",
    "daejun": "Dark-eyed Junco",
    "houfin": "House Finch",
    "houspa": "House Sparrow",
    "norfli": "Northern Flicker",
    "rewbla": "Red-winged Blackbird",
    "sonspa": "Song Sparrow",
    "spotow": "Spotted Towhee",
    "whcspa": "White-crowned Sparrow"
}

# Empty dictionary to hold spectrograms.
full_bird = {}

# Generate a dictionary for spectrograms. 
# This will make it easier to prep when doing a binary model.
for name in f.keys():
    # Transpose data from each species to N, 128, 517. 
    full_bird[name] = f[name][:].transpose((2,0,1))
    print(f"{bird_names[name]} ({name}):  {full_bird[name].shape}")
    
American Crow (amecro):  (66, 128, 517)
American Robin (amerob):  (172, 128, 517)
Bewick's Wren (bewwre):  (144, 128, 517)
Black-capped Chickadee (bkcchi):  (45, 128, 517)
Dark-eyed Junco (daejun):  (125, 128, 517)
House Finch (houfin):  (84, 128, 517)
House Sparrow (houspa):  (630, 128, 517)
Northern Flicker (norfli):  (37, 128, 517)
Red-winged Blackbird (rewbla):  (187, 128, 517)
Song Sparrow (sonspa):  (263, 128, 517)
Spotted Towhee (spotow):  (137, 128, 517)
White-crowned Sparrow (whcspa):  (91, 128, 517)
In [9]:
# Accuracy and loss graphs
def plot_training_history(history, title):
    plt.figure(figsize=(12, 5))

    # Accuracy plot
    plt.subplot(1, 2, 1)
    plt.plot(history.history['accuracy'], label='Train Accuracy')
    plt.plot(history.history['val_accuracy'], label='Val Accuracy')
    plt.title(f'{title} Accuracy Over Epochs')
    plt.xlabel('Epoch')
    plt.ylabel('Accuracy')
    plt.legend(['Accuracy', 'Validation Accuracy'], loc='upper right')
    plt.grid(True)

    # Loss plot
    plt.subplot(1, 2, 2)
    plt.plot(history.history['loss'], label='Train Loss')
    plt.plot(history.history['val_loss'], label='Val Loss')
    plt.title(f'{title} Loss Over Epochs')
    plt.xlabel('Epoch')
    plt.ylabel('Loss')
    plt.legend(['Loss', 'Validation Loss'], loc='upper right')
    plt.grid(True)

    plt.tight_layout()
    plt.show()

def model_matrics(history, title, batch_size):
    print(f'\n{title}')
    
    final_train_acc = history.history['accuracy'][-1]
    final_val_acc = history.history['val_accuracy'][-1]
    final_train_loss = history.history['loss'][-1]
    final_val_loss = history.history['val_loss'][-1] 
    total_epochs = len(history.history['loss'])

    print(f"Epochs Trained: {total_epochs}")
    if batch_size is not None:
        print(f"Batch Size: {batch_size}")
    
    print(f"Final Training Accuracy: {final_train_acc:.3f}")
    print(f"Final Validation Accuracy: {final_val_acc:.3f}")
    print(f"Final Training Loss: {final_train_loss:.3f}")
    print(f"Final Validation Loss: {final_val_loss:.3f}")
In [197]:
# Label Binarizer to transform y values to 0 and 1. 
lb = LabelBinarizer()

# Function to generate data for binary model
def generate_binomial_data(species_list):

    # Empty lists to hold data
    X_list = []
    y_list = []
    
    # Loop through list containing species and append lists
    for species in species_list:
        specs = full_bird[species]
        X_list.append(specs)
        y_list.extend([species] * specs.shape[0])

    # Concatenate a list of spectrograms
    X = np.concatenate(X_list, axis=0)
    y = np.array(y_list)

    # Adding channel dimension
    X = X[..., np.newaxis]

    # Convert categorical labels for species to 0, 1. 
    y = lb.fit_transform(y).ravel()

    # Return prep data for training and testing. 
    # Stratifying on y, since I want equal representation in both training and test sets. 
    return train_test_split(X, y, test_size=0.33, random_state=13, stratify=y)

# Generate data for a binary model 
X_train, X_test, y_train, y_test = generate_binomial_data(['bewwre', 'spotow'])
In [199]:
# Evaluate contents of y_train and y_test
unique, counts = np.unique(y_train, return_counts=True)
print('y_train contents: ')
for label, count in zip(unique, counts):
    print(f"Class {label}: {count} samples")

unique, counts = np.unique(y_test, return_counts=True)
print('\ny_test contents: ')
for label, count in zip(unique, counts):
    print(f"Class {label}: {count} samples")
y_train contents: 
Class 0: 96 samples
Class 1: 92 samples

y_test contents: 
Class 0: 48 samples
Class 1: 45 samples
In [201]:
# Stop training if val_loss does not improve for 3 epochs and returns weights to best val_loss model
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

# Define model seq_model_simple(title, units, activation function, shape)
def seq_model_simple(title, units, act, shape):
    print(f'{title} (Dense Units: {units}, Activation Function: {act})')
    model = Sequential(
    [
    Dense(units=units, activation=act, input_shape=(shape,)),
    Dense(units=units, activation=act),
    Dense(units=1, activation='sigmoid')
    ])
    return model

# Define model seq_model_simple(title, units, activation function, shape, drop)
def seq_model_simple_drop(title, units, act, shape, drop):
    print(f'{title} (Dense Units: {units}, Activation Function: {act}, Dropout: {drop})')
    model = Sequential([
    Dense(units=units, activation=act, input_shape=(shape,)),
    Dropout(drop),
    Dense(units=units, activation=act),
    Dropout(drop),
    Dense(units=1, activation='sigmoid')])
    return model

# Compile model with seq_model_compile(title, defined model, optimization function, loss function, metric)
def seq_model_compile(title, model, opt, l, m):
    print(f'{title} (Optimizer Function: {opt}, Loss: {l}, Metric: {m})')
    model.compile(optimizer=opt
              , loss= l
              , metrics=[m])
    return model

# Test model simple_model(name, model, X_train, X_test, y_train, y_test, epoch, batch_size, early_stop)
def simple_model(title, model, xtrain, xtest, ytrain, ytest, e, b, stop):
    print(f'{title} (Epoch: {e}, Batch Size: {b})')
    history = model.fit(xtrain, ytrain, 
                    epochs=e, 
                    batch_size=b,
                    validation_data=(xtest, ytest),
                    callbacks=[stop],
                   verbose=0)
    name = f'{title} (Batch Size={b}, Epoch={e})'
    model_matrics(history, name, b)
    loss, acc = model.evaluate(xtest, ytest, verbose=0)
    print(f"Test Accuracy (from model.evaluate): {acc:.3f}")
    print(f"Trainable Params: {model.count_params()}")
    plot_training_history(history, title)
    return history
In [203]:
# MLP model expects a 1D vector, reshaping X_train and X_test
X_train_flat = X_train.reshape(X_train.shape[0] ,-1)
X_test_flat = X_test.reshape(X_test.shape[0] ,-1)

# Params to define model
input_dim = 128 * 517
u = [16, 32, 64]
act = 'relu'

# Params to compile model
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'

# Test model
title = 'Simple MLP'
b_sizes = [32, 64, 500]
epoch = 1000

results = []

# Loop to test different 
for u_size in u:
    print(f"\n=== Training {title} with Unit Size: {u_size} ===")
    for b_size in b_sizes:
        print(f"\n=== Training {title} with Batch Size: {b_size} ===")
        
        model = seq_model_simple(title, u_size, act, input_dim)
        model = seq_model_compile(title, model, opt, l, m)
        
        history = simple_model(title, 
                     model, 
                     X_train_flat, 
                     X_test_flat, 
                     y_train, 
                     y_test, 
                     epoch, 
                     b_size, 
                     stop=early_stop)
       
        y_pred_probs = model.predict(X_test_flat)
        y_pred = (y_pred_probs > 0.5).astype(int).flatten()

        cm = confusion_matrix(y_test, y_pred)
        tn, fp, fn, tp = cm.ravel()
        acc = accuracy_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred, zero_division=0)
        recall = recall_score(y_test, y_pred, zero_division=0)
        f1 = f1_score(y_test, y_pred, zero_division=0)

        results.append({
            'Units': u_size,
            'Batch Size': b_size,
            'Epochs Train': len(history.history['loss']),
            'TP': tp,
            'TN': tn,
            'FP': fp,
            'FN': fn,
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

results_df = pd.DataFrame(results)
display(results_df)
=== Training Simple MLP with Unit Size: 16 ===

=== Training Simple MLP with Batch Size: 32 ===
Simple MLP (Dense Units: 16, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=32, Epoch=1000)
Epochs Trained: 61
Batch Size: 32
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.505
Final Training Loss: 0.693
Final Validation Loss: 0.694
Test Accuracy (from model.evaluate): 0.505
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Batch Size: 64 ===
Simple MLP (Dense Units: 16, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=64, Epoch=1000)
Epochs Trained: 11
Batch Size: 64
Final Training Accuracy: 0.479
Final Validation Accuracy: 0.484
Final Training Loss: 23.222
Final Validation Loss: 24.531
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Batch Size: 500 ===
Simple MLP (Dense Units: 16, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=500, Epoch=1000)
Epochs Trained: 146
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.516
Final Training Loss: 0.693
Final Validation Loss: 0.693
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Unit Size: 32 ===

=== Training Simple MLP with Batch Size: 32 ===
Simple MLP (Dense Units: 32, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=32, Epoch=1000)
Epochs Trained: 8
Batch Size: 32
Final Training Accuracy: 0.479
Final Validation Accuracy: 0.516
Final Training Loss: 318.174
Final Validation Loss: 905.095
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Batch Size: 64 ===
Simple MLP (Dense Units: 32, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=64, Epoch=1000)
Epochs Trained: 6
Batch Size: 64
Final Training Accuracy: 0.457
Final Validation Accuracy: 0.484
Final Training Loss: 698.699
Final Validation Loss: 230.969
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP with Batch Size: 500 ===
Simple MLP (Dense Units: 32, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=500, Epoch=1000)
Epochs Trained: 5
Batch Size: 500
Final Training Accuracy: 0.489
Final Validation Accuracy: 0.516
Final Training Loss: 227.642
Final Validation Loss: 1577.145
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP with Unit Size: 64 ===

=== Training Simple MLP with Batch Size: 32 ===
Simple MLP (Dense Units: 64, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=32, Epoch=1000)
Epochs Trained: 24
Batch Size: 32
Final Training Accuracy: 0.495
Final Validation Accuracy: 0.516
Final Training Loss: 25.170
Final Validation Loss: 14.566
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP with Batch Size: 64 ===
Simple MLP (Dense Units: 64, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=64, Epoch=1000)
Epochs Trained: 7
Batch Size: 64
Final Training Accuracy: 0.463
Final Validation Accuracy: 0.516
Final Training Loss: 143.879
Final Validation Loss: 311.193
Test Accuracy (from model.evaluate): 0.581
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP with Batch Size: 500 ===
Simple MLP (Dense Units: 64, Activation Function: relu)
Simple MLP (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Batch Size=500, Epoch=1000)
Epochs Trained: 10
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.484
Final Training Loss: 205.062
Final Validation Loss: 2061.232
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 
Units Batch Size Epochs Train TP TN FP FN Accuracy Precision Recall F1
0 16 32 61 0 47 1 45 0.505376 0.000000 0.0 0.000000
1 16 64 11 45 0 48 0 0.483871 0.483871 1.0 0.652174
2 16 500 146 0 48 0 45 0.516129 0.000000 0.0 0.000000
3 32 32 8 0 48 0 45 0.516129 0.000000 0.0 0.000000
4 32 64 6 45 0 48 0 0.483871 0.483871 1.0 0.652174
5 32 500 5 0 48 0 45 0.516129 0.000000 0.0 0.000000
6 64 32 24 0 48 0 45 0.516129 0.000000 0.0 0.000000
7 64 64 7 9 45 3 36 0.580645 0.750000 0.2 0.315789
8 64 500 10 45 0 48 0 0.483871 0.483871 1.0 0.652174
In [205]:
# Scaled images to the [0, 1] range
X_train_scaled = X_train / 255
X_test_scaled = X_test / 255

# 1D vectors
X_train_flat = X_train_scaled.reshape(X_train_scaled.shape[0], -1)
X_test_flat = X_test_scaled.reshape(X_test_scaled.shape[0], -1)

# Params to define model
input_dim = 128 * 517
u = [16, 32, 64]
act = 'relu'

# Params to compile model
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'

# Test model
title = 'Simple MLP (Scaled)'
b_sizes = [32, 64, 500]
epoch = 1000

results = []

# Loop to test different 
for u_size in u:
    print(f"\n=== Training {title} with Unit Size: {u_size} ===")
    for b_size in b_sizes:
        print(f"\n=== Training {title} with Batch Size: {b_size} ===")
        
        model = seq_model_simple(title, u_size, act, input_dim)
        model = seq_model_compile(title, model, opt, l, m)
        
        history = simple_model(title, 
                     model, 
                     X_train_flat, 
                     X_test_flat, 
                     y_train, 
                     y_test, 
                     epoch, 
                     b_size, 
                     stop=early_stop)
       
        y_pred_probs = model.predict(X_test_flat)
        y_pred = (y_pred_probs > 0.5).astype(int).flatten()

        cm = confusion_matrix(y_test, y_pred)
        tn, fp, fn, tp = cm.ravel()
        acc = accuracy_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred, zero_division=0)
        recall = recall_score(y_test, y_pred, zero_division=0)
        f1 = f1_score(y_test, y_pred, zero_division=0)

        results.append({
            'Units': u_size,
            'Batch Size': b_size,
            'Epochs Train': len(history.history['loss']),
            'TP': tp,
            'TN': tn,
            'FP': fp,
            'FN': fn,
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

results_df = pd.DataFrame(results)
display(results_df)
=== Training Simple MLP (Scaled) with Unit Size: 16 ===

=== Training Simple MLP (Scaled) with Batch Size: 32 ===
Simple MLP (Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 10
Batch Size: 32
Final Training Accuracy: 0.441
Final Validation Accuracy: 0.516
Final Training Loss: 1.665
Final Validation Loss: 1.528
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 64 ===
Simple MLP (Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 19
Batch Size: 64
Final Training Accuracy: 0.585
Final Validation Accuracy: 0.516
Final Training Loss: 0.687
Final Validation Loss: 0.697
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 500 ===
Simple MLP (Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 15
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.484
Final Training Loss: 1.861
Final Validation Loss: 5.392
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Unit Size: 32 ===

=== Training Simple MLP (Scaled) with Batch Size: 32 ===
Simple MLP (Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 5
Batch Size: 32
Final Training Accuracy: 0.457
Final Validation Accuracy: 0.516
Final Training Loss: 1.743
Final Validation Loss: 1.606
Test Accuracy (from model.evaluate): 0.527
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 64 ===
Simple MLP (Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 67
Batch Size: 64
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.516
Final Training Loss: 0.693
Final Validation Loss: 0.693
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 500 ===
Simple MLP (Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 6
Batch Size: 500
Final Training Accuracy: 0.521
Final Validation Accuracy: 0.484
Final Training Loss: 1.705
Final Validation Loss: 15.005
Test Accuracy (from model.evaluate): 0.570
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Unit Size: 64 ===

=== Training Simple MLP (Scaled) with Batch Size: 32 ===
Simple MLP (Scaled) (Dense Units: 64, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 5
Batch Size: 32
Final Training Accuracy: 0.543
Final Validation Accuracy: 0.516
Final Training Loss: 1.751
Final Validation Loss: 3.042
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 64 ===
Simple MLP (Scaled) (Dense Units: 64, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 9
Batch Size: 64
Final Training Accuracy: 0.457
Final Validation Accuracy: 0.516
Final Training Loss: 0.972
Final Validation Loss: 1.248
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP (Scaled) with Batch Size: 500 ===
Simple MLP (Scaled) (Dense Units: 64, Activation Function: relu)
Simple MLP (Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Scaled) (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 6
Batch Size: 500
Final Training Accuracy: 0.489
Final Validation Accuracy: 0.516
Final Training Loss: 1.729
Final Validation Loss: 1.331
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 
Units Batch Size Epochs Train TP TN FP FN Accuracy Precision Recall F1
0 16 32 10 1 47 1 44 0.516129 0.500000 0.022222 0.042553
1 16 64 19 0 48 0 45 0.516129 0.000000 0.000000 0.000000
2 16 500 15 0 48 0 45 0.516129 0.000000 0.000000 0.000000
3 32 32 5 1 48 0 44 0.526882 1.000000 0.022222 0.043478
4 32 64 67 0 48 0 45 0.516129 0.000000 0.000000 0.000000
5 32 500 6 10 43 5 35 0.569892 0.666667 0.222222 0.333333
6 64 32 5 0 48 0 45 0.516129 0.000000 0.000000 0.000000
7 64 64 9 45 0 48 0 0.483871 0.483871 1.000000 0.652174
8 64 500 6 45 0 48 0 0.483871 0.483871 1.000000 0.652174
In [207]:
# 1D vectors
X_train_flat = X_train.reshape(X_train.shape[0] ,-1)
X_test_flat = X_test.reshape(X_test.shape[0] ,-1)

mean = np.mean(X_train_flat)
std = np.std(X_train_flat)

X_train_flat_scaled = (X_train_flat - mean) / std
X_test_flat_scaled = (X_test_flat - mean) / std


# Params to define model
input_dim = 128 * 517
u = [16, 32, 64]
act = 'relu'

# Params to compile model
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'

# Test model
title = 'Simple MLP (Z Scaled)'
b_sizes = [32, 64, 500]
epoch = 1000

results = []

# Loop to test different 
for u_size in u:
    print(f"\n=== Training {title} with Unit Size: {u_size} ===")
    for b_size in b_sizes:
        print(f"\n=== Training {title} with Batch Size: {b_size} ===")
        
        model = seq_model_simple(title, u_size, act, input_dim)
        model = seq_model_compile(title, model, opt, l, m)
        
        history = simple_model(title, 
                     model, 
                     X_train_flat, 
                     X_test_flat, 
                     y_train, 
                     y_test, 
                     epoch, 
                     b_size, 
                     stop=early_stop)
       
        y_pred_probs = model.predict(X_test_flat)
        y_pred = (y_pred_probs > 0.5).astype(int).flatten()

        cm = confusion_matrix(y_test, y_pred)
        tn, fp, fn, tp = cm.ravel()
        acc = accuracy_score(y_test, y_pred)
        precision = precision_score(y_test, y_pred, zero_division=0)
        recall = recall_score(y_test, y_pred, zero_division=0)
        f1 = f1_score(y_test, y_pred, zero_division=0)

        results.append({
            'Units': u_size,
            'Batch Size': b_size,
            'Epochs Train': len(history.history['loss']),
            'TP': tp,
            'TN': tn,
            'FP': fp,
            'FN': fn,
            'Accuracy': acc,
            'Precision': precision,
            'Recall': recall,
            'F1': f1
        })

results_df = pd.DataFrame(results)
display(results_df)
=== Training Simple MLP (Z Scaled) with Unit Size: 16 ===

=== Training Simple MLP (Z Scaled) with Batch Size: 32 ===
Simple MLP (Z Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 11
Batch Size: 32
Final Training Accuracy: 0.516
Final Validation Accuracy: 0.516
Final Training Loss: 406.022
Final Validation Loss: 766.301
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 64 ===
Simple MLP (Z Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 6
Batch Size: 64
Final Training Accuracy: 0.489
Final Validation Accuracy: 0.516
Final Training Loss: 8.166
Final Validation Loss: 7.217
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 500 ===
Simple MLP (Z Scaled) (Dense Units: 16, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 9
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.484
Final Training Loss: 148.349
Final Validation Loss: 378.533
Test Accuracy (from model.evaluate): 0.462
Trainable Params: 1059121
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP (Z Scaled) with Unit Size: 32 ===

=== Training Simple MLP (Z Scaled) with Batch Size: 32 ===
Simple MLP (Z Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 8
Batch Size: 32
Final Training Accuracy: 0.489
Final Validation Accuracy: 0.484
Final Training Loss: 46.705
Final Validation Loss: 46.867
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 64 ===
Simple MLP (Z Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 8
Batch Size: 64
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.516
Final Training Loss: 363.927
Final Validation Loss: 73.287
Test Accuracy (from model.evaluate): 0.430
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 500 ===
Simple MLP (Z Scaled) (Dense Units: 32, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 7
Batch Size: 500
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.484
Final Training Loss: 191.046
Final Validation Loss: 471.429
Test Accuracy (from model.evaluate): 0.484
Trainable Params: 2118753
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 6ms/step 

=== Training Simple MLP (Z Scaled) with Unit Size: 64 ===

=== Training Simple MLP (Z Scaled) with Batch Size: 32 ===
Simple MLP (Z Scaled) (Dense Units: 64, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 32)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=32, Epoch=1000)
Epochs Trained: 62
Batch Size: 32
Final Training Accuracy: 0.511
Final Validation Accuracy: 0.516
Final Training Loss: 0.693
Final Validation Loss: 0.693
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 8ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 64 ===
Simple MLP (Z Scaled) (Dense Units: 64, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 64)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=64, Epoch=1000)
Epochs Trained: 14
Batch Size: 64
Final Training Accuracy: 0.532
Final Validation Accuracy: 0.484
Final Training Loss: 206.358
Final Validation Loss: 167.539
Test Accuracy (from model.evaluate): 0.495
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 

=== Training Simple MLP (Z Scaled) with Batch Size: 500 ===
Simple MLP (Z Scaled) (Dense Units: 64, Activation Function: relu)
Simple MLP (Z Scaled) (Optimizer Function: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
Simple MLP (Z Scaled) (Epoch: 1000, Batch Size: 500)
/opt/anaconda3/lib/python3.12/site-packages/keras/src/layers/core/dense.py:87: UserWarning: Do not pass an `input_shape`/`input_dim` argument to a layer. When using Sequential models, prefer using an `Input(shape)` object as the first layer in the model instead.
  super().__init__(activity_regularizer=activity_regularizer, **kwargs)
Simple MLP (Z Scaled) (Batch Size=500, Epoch=1000)
Epochs Trained: 5
Batch Size: 500
Final Training Accuracy: 0.489
Final Validation Accuracy: 0.516
Final Training Loss: 422.180
Final Validation Loss: 1211.839
Test Accuracy (from model.evaluate): 0.516
Trainable Params: 4239553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 7ms/step 
Units Batch Size Epochs Train TP TN FP FN Accuracy Precision Recall F1
0 16 32 11 45 0 48 0 0.483871 0.483871 1.000000 0.652174
1 16 64 6 45 0 48 0 0.483871 0.483871 1.000000 0.652174
2 16 500 9 11 32 16 34 0.462366 0.407407 0.244444 0.305556
3 32 32 8 45 0 48 0 0.483871 0.483871 1.000000 0.652174
4 32 64 8 26 14 34 19 0.430108 0.433333 0.577778 0.495238
5 32 500 7 45 0 48 0 0.483871 0.483871 1.000000 0.652174
6 64 32 62 0 48 0 45 0.516129 0.000000 0.000000 0.000000
7 64 64 14 45 1 47 0 0.494624 0.489130 1.000000 0.656934
8 64 500 5 0 48 0 45 0.516129 0.000000 0.000000 0.000000
In [82]:
# Build CNN model
def build_cnn_model(name, layers, channels, k_size, pool, act, drop, dense, u):
    print(f'{name} with {layers} layer(s), {channels} channels, {k_size} kernel size, {pool} pool size, {act} activation, {drop} dropout, {dense} hidden dense layer(s)')
    
    model = Sequential()
    model.add(Input(shape=(128, 517, 1)))
    model.add(Conv2D(filters=channels, kernel_size=k_size, padding='same', activation=act))
    model.add(MaxPooling2D(pool_size=pool))

    for i in range(1, layers):
        model.add(Conv2D(filters=channels * (2**i), kernel_size=k_size, padding='same', activation=act))
        model.add(MaxPooling2D(pool_size=pool))

    model.add(Flatten())
    model.add(Dropout(drop))
    
    units = u
    for j in range(dense):
        units = int(units)
        model.add(Dense(units=units, activation=act))
        if drop > 0:
            model.add(Dropout(drop)) 
        units = units/2

    model.add(Dense(1, activation='sigmoid'))
    return model

# Compile function
def seq_model_compile(title, model, opt, l, m):
    print(f'{title} Compiled (Optimizer: {opt}, Loss: {l}, Metric: {m})')
    model.compile(optimizer=opt, loss=l, metrics=[m])
    return model

# Train and evaluate
def cnn_model(title, model, xtrain, xtest, ytrain, ytest, e, b, stop):
    print(f'{title} Tested (Optimizer: {opt}, Loss: {l}, Metric: {m})')
    history = model.fit(xtrain, ytrain, 
                        epochs=e, 
                        batch_size=b,
                        validation_data=(xtest, ytest),
                        callbacks=[stop],
                        verbose=0)
    name = f'{title} (Batch Size={b}, Epoch={e})'
    model_matrics(history, name, b)
    loss, acc = model.evaluate(xtest, ytest, verbose=0)
    print(f"Test Accuracy: {acc:.3f}")
    print(f"Trainable Params: {model.count_params()}")
    plot_training_history(history, title)
    return history

def cnn_metrics(model, y_test, X_test_scaled, layers, channels, lb):
    y_pred_probs = model.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    label_map = f"{lb.classes_[0]}=0, {lb.classes_[1]}=1"

    results_df = pd.DataFrame([{
        'Label Mapping': label_map,
        'Layers': layers,
        'Channels': channels,
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    }])
    
    display(results_df)
In [39]:
# Generate data for a binary model
X_train, X_test, y_train, y_test = generate_binomial_data(['bewwre', 'spotow'])
X_train_scaled = X_train / 255.0
X_test_scaled = X_test / 255.0

# Params
layer_list = [1]
channels = 16
k_size = (3, 3)
pool = (2, 2)
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for layers in layer_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Scaled Data with 1 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 12
Batch Size: 32
Final Training Accuracy: 0.697
Final Validation Accuracy: 0.516
Final Training Loss: 0.570
Final Validation Loss: 0.822
Test Accuracy: 0.602
Trainable Params: 264353
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 25ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Scaled Data 1 16 (3, 3) (2, 2) 0.5 0 128 32 12 264353 22 34 14 23 0.602151 0.611111 0.488889 0.54321
In [43]:
mean = np.mean(X_train)
std = np.std(X_train)

X_train_scaled = (X_train - mean) / std
X_test_scaled = (X_test - mean) / std
In [45]:
# Params
layer_list = [1]
channels = 16
k_size = (3, 3)
pool = (2, 2)
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for layers in layer_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 1 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 18
Batch Size: 32
Final Training Accuracy: 0.676
Final Validation Accuracy: 0.559
Final Training Loss: 1.025
Final Validation Loss: 0.838
Test Accuracy: 0.645
Trainable Params: 264353
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 25ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 1 16 (3, 3) (2, 2) 0.5 0 128 32 18 264353 28 32 16 17 0.645161 0.636364 0.622222 0.629213

Observation:¶

Z-scaling the data improved result (Accuracy, Precision, Recall, F1 Score). All model tuning will be done with Z-scaled data

CNN Hyperparameter Optimization¶

Layer Optimization¶

In [50]:
layer_list = [1, 2, 3]
channels = 16
k_size = (3, 3)
pool = (2, 2)
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for layers in layer_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 1 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 8
Batch Size: 32
Final Training Accuracy: 0.686
Final Validation Accuracy: 0.484
Final Training Loss: 1.313
Final Validation Loss: 4.232
Test Accuracy: 0.581
Trainable Params: 264353
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 25ms/step
CNN Z-Scaled Data with 2 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 12
Batch Size: 32
Final Training Accuracy: 0.936
Final Validation Accuracy: 0.570
Final Training Loss: 0.231
Final Validation Loss: 0.675
Test Accuracy: 0.634
Trainable Params: 136897
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 37ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 15
Batch Size: 32
Final Training Accuracy: 0.851
Final Validation Accuracy: 0.613
Final Training Loss: 0.330
Final Validation Loss: 0.818
Test Accuracy: 0.645
Trainable Params: 88833
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 44ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 1 16 (3, 3) (2, 2) 0.5 0 128 32 8 264353 26 28 20 19 0.580645 0.565217 0.577778 0.571429
1 CNN Z-Scaled Data 2 16 (3, 3) (2, 2) 0.5 0 128 32 12 136897 25 34 14 20 0.634409 0.641026 0.555556 0.595238
2 CNN Z-Scaled Data 3 16 (3, 3) (2, 2) 0.5 0 128 32 15 88833 19 41 7 26 0.645161 0.730769 0.422222 0.535211

Observation: 3 layers gave us good accuracy and Precision, but lower F1 scores. We will proceed with three layers for now¶

Pool Optimization¶

In [59]:
layers = 3
channels = 16
k_size = (3, 3)
pool_list = [(2, 2), (3,3), (4,4), (5,5)]
act = 'relu'
dropout = 0.5
dense_layers = 0
units = 128
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for pool in pool_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (2, 2) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.915
Final Validation Accuracy: 0.527
Final Training Loss: 0.316
Final Validation Loss: 1.285
Test Accuracy: 0.624
Trainable Params: 88833
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 49ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 7
Batch Size: 32
Final Training Accuracy: 0.739
Final Validation Accuracy: 0.667
Final Training Loss: 0.566
Final Validation Loss: 0.614
Test Accuracy: 0.720
Trainable Params: 28161
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 34ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (4, 4) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 13
Batch Size: 32
Final Training Accuracy: 0.739
Final Validation Accuracy: 0.656
Final Training Loss: 0.557
Final Validation Loss: 0.619
Test Accuracy: 0.688
Trainable Params: 24321
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 32ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (5, 5) pool size, relu activation, 0.5 dropout, 0 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.649
Final Validation Accuracy: 0.591
Final Training Loss: 0.609
Final Validation Loss: 0.688
Test Accuracy: 0.613
Trainable Params: 23553
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 29ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (3, 3) (2, 2) 0.5 0 128 32 14 88833 27 31 17 18 0.623656 0.613636 0.600000 0.606742
1 CNN Z-Scaled Data 3 16 (3, 3) (3, 3) 0.5 0 128 32 7 28161 29 38 10 16 0.720430 0.743590 0.644444 0.690476
2 CNN Z-Scaled Data 3 16 (3, 3) (4, 4) 0.5 0 128 32 13 24321 25 39 9 20 0.688172 0.735294 0.555556 0.632911
3 CNN Z-Scaled Data 3 16 (3, 3) (5, 5) 0.5 0 128 32 14 23553 17 40 8 28 0.612903 0.680000 0.377778 0.485714

Observation: Higher performance of pool size (3,3)¶

Dense layer Units Optmization¶

In [68]:
layers = 3
channels = 16
k_size = (3, 3)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units_list = [512, 256, 128]
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for units in units_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.803
Final Validation Accuracy: 0.538
Final Training Loss: 0.454
Final Validation Loss: 0.810
Test Accuracy: 0.645
Trainable Params: 2514689
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 38ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 15
Batch Size: 32
Final Training Accuracy: 0.755
Final Validation Accuracy: 0.591
Final Training Loss: 0.497
Final Validation Loss: 0.641
Test Accuracy: 0.699
Trainable Params: 1268993
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 38ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 9
Batch Size: 32
Final Training Accuracy: 0.665
Final Validation Accuracy: 0.570
Final Training Loss: 0.559
Final Validation Loss: 0.628
Test Accuracy: 0.634
Trainable Params: 646145
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 36ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (3, 3) (3, 3) 0.5 1 512 32 14 2514689 24 36 12 21 0.645161 0.666667 0.533333 0.592593
1 CNN Z-Scaled Data 3 16 (3, 3) (3, 3) 0.5 1 256 32 15 1268993 30 35 13 15 0.698925 0.697674 0.666667 0.681818
2 CNN Z-Scaled Data 3 16 (3, 3) (3, 3) 0.5 1 128 32 9 646145 15 44 4 30 0.634409 0.789474 0.333333 0.468750

Observation: 256 units in the hidden dense layer gave us higher accuracy, better recall and F1 score.¶

K Size Optimization¶

In [72]:
layers = 3
channels = 16
k_size_list = [(3, 3), (4,4), (5,5)]
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 32
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for k_size in k_size_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (3, 3) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 16
Batch Size: 32
Final Training Accuracy: 0.819
Final Validation Accuracy: 0.570
Final Training Loss: 0.444
Final Validation Loss: 0.675
Test Accuracy: 0.667
Trainable Params: 1268993
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 37ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (4, 4) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 11
Batch Size: 32
Final Training Accuracy: 0.676
Final Validation Accuracy: 0.527
Final Training Loss: 0.557
Final Validation Loss: 0.667
Test Accuracy: 0.645
Trainable Params: 1287025
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 43ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 15
Batch Size: 32
Final Training Accuracy: 0.782
Final Validation Accuracy: 0.656
Final Training Loss: 0.518
Final Validation Loss: 0.632
Test Accuracy: 0.677
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 48ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (3, 3) (3, 3) 0.5 1 256 32 16 1268993 27 35 13 18 0.666667 0.675000 0.600000 0.635294
1 CNN Z-Scaled Data 3 16 (4, 4) (3, 3) 0.5 1 256 32 11 1287025 18 42 6 27 0.645161 0.750000 0.400000 0.521739
2 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 32 15 1310209 29 34 14 16 0.677419 0.674419 0.644444 0.659091

Observation: (5,5) performed best in terms of accuracy, recall and F1 score¶

Batch Optimization¶

In [76]:
layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch_list = [16, 32, 64]
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for batch in batch_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=16, Epoch=100)
Epochs Trained: 11
Batch Size: 16
Final Training Accuracy: 0.745
Final Validation Accuracy: 0.591
Final Training Loss: 0.496
Final Validation Loss: 0.671
Test Accuracy: 0.613
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 50ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=32, Epoch=100)
Epochs Trained: 14
Batch Size: 32
Final Training Accuracy: 0.771
Final Validation Accuracy: 0.602
Final Training Loss: 0.545
Final Validation Loss: 0.636
Test Accuracy: 0.613
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 49ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 22
Batch Size: 64
Final Training Accuracy: 0.745
Final Validation Accuracy: 0.548
Final Training Loss: 0.498
Final Validation Loss: 0.675
Test Accuracy: 0.688
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 51ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 16 11 1310209 18 39 9 27 0.612903 0.666667 0.4 0.500000
1 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 32 14 1310209 18 39 9 27 0.612903 0.666667 0.4 0.500000
2 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 64 22 1310209 36 28 20 9 0.688172 0.642857 0.8 0.712871

Observation: Batch size of 64 provided better accuracy, recall, and F1 scores.¶

Channel Optimization¶

In [79]:
layers = 3
channels_list = [16, 32]
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch_list = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for channels in channels_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 20
Batch Size: 64
Final Training Accuracy: 0.681
Final Validation Accuracy: 0.591
Final Training Loss: 0.586
Final Validation Loss: 0.704
Test Accuracy: 0.634
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 47ms/step
CNN Z-Scaled Data with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 20
Batch Size: 64
Final Training Accuracy: 0.617
Final Validation Accuracy: 0.602
Final Training Loss: 0.655
Final Validation Loss: 0.644
Test Accuracy: 0.613
Trainable Params: 2747905
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 88ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 64 20 1310209 16 43 5 29 0.634409 0.761905 0.355556 0.484848
1 CNN Z-Scaled Data 3 32 (5, 5) (3, 3) 0.5 1 256 64 20 2747905 16 41 7 29 0.612903 0.695652 0.355556 0.470588

Observation: Keep the Number of chanels the same.¶

Dense Layer tuning¶

In [93]:
layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.5
dense_layers_list = [1, 2, 3]
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)

cnn_results = []

for dense_layers in dense_layers_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 26
Batch Size: 64
Final Training Accuracy: 0.718
Final Validation Accuracy: 0.505
Final Training Loss: 0.564
Final Validation Loss: 1.049
Test Accuracy: 0.634
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 54ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 2 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 20
Batch Size: 64
Final Training Accuracy: 0.707
Final Validation Accuracy: 0.667
Final Training Loss: 0.590
Final Validation Loss: 0.607
Test Accuracy: 0.656
Trainable Params: 1342977
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 48ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 3 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 18
Batch Size: 64
Final Training Accuracy: 0.601
Final Validation Accuracy: 0.538
Final Training Loss: 0.646
Final Validation Loss: 0.712
Test Accuracy: 0.602
Trainable Params: 1351169
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 50ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 64 26 1310209 21 38 10 24 0.634409 0.677419 0.466667 0.552632
1 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 2 256 64 20 1342977 24 37 11 21 0.655914 0.685714 0.533333 0.600000
2 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 3 256 64 18 1351169 18 38 10 27 0.602151 0.642857 0.400000 0.493151

Observation: Dense layers should be 1 at 256 units¶

Dropout Optimization¶

In [97]:
layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout_list = [0.5, 0.4, 0.3]
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

cnn_results = []

for dropout in dropout_list:
    model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, l, m)
    history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test_scaled)
    y_pred = (y_pred_probs > 0.5).astype(int).flatten()

    cm = confusion_matrix(y_test, y_pred)
    tn, fp, fn, tp = cm.ravel()
    acc = accuracy_score(y_test, y_pred)
    precision = precision_score(y_test, y_pred, zero_division=0)
    recall = recall_score(y_test, y_pred, zero_division=0)
    f1 = f1_score(y_test, y_pred, zero_division=0)

    cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.5 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 25
Batch Size: 64
Final Training Accuracy: 0.777
Final Validation Accuracy: 0.624
Final Training Loss: 0.511
Final Validation Loss: 0.632
Test Accuracy: 0.624
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 42ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.4 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 21
Batch Size: 64
Final Training Accuracy: 0.734
Final Validation Accuracy: 0.581
Final Training Loss: 0.583
Final Validation Loss: 0.661
Test Accuracy: 0.591
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 46ms/step
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.3 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 25
Batch Size: 64
Final Training Accuracy: 0.771
Final Validation Accuracy: 0.613
Final Training Loss: 0.503
Final Validation Loss: 0.626
Test Accuracy: 0.699
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 48ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.5 1 256 64 25 1310209 26 32 16 19 0.623656 0.619048 0.577778 0.597701
1 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.4 1 256 64 21 1310209 28 27 21 17 0.591398 0.571429 0.622222 0.595745
2 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.3 1 256 64 25 1310209 33 32 16 12 0.698925 0.673469 0.733333 0.702128

Observation: Dropout at 0.3 improved accuracy, recall and F1 score.¶

Final Model¶

In [106]:
layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.3
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

cnn_results = []


model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
model_cnn = seq_model_compile(title, model, opt, l, m)
history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)
y_pred_probs = model_cnn.predict(X_test_scaled)
y_pred = (y_pred_probs > 0.5).astype(int).flatten()
cm = confusion_matrix(y_test, y_pred)
tn, fp, fn, tp = cm.ravel()
acc = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, zero_division=0)
recall = recall_score(y_test, y_pred, zero_division=0)
f1 = f1_score(y_test, y_pred, zero_division=0)
cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.3 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 16
Batch Size: 64
Final Training Accuracy: 0.723
Final Validation Accuracy: 0.634
Final Training Loss: 0.559
Final Validation Loss: 0.665
Test Accuracy: 0.613
Trainable Params: 1310209
No description has been provided for this image
3/3 ━━━━━━━━━━━━━━━━━━━━ 0s 52ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.3 1 256 64 16 1310209 29 28 20 16 0.612903 0.591837 0.644444 0.617021

Test on different set of birds¶

In [111]:
X_train, X_test, y_train, y_test = generate_binomial_data(['amecro', 'houspa'])
mean = np.mean(X_train)
std = np.std(X_train)

X_train_scaled = (X_train - mean) / std
X_test_scaled = (X_test - mean) / std

layers = 3
channels = 16
k_size = (5,5)
pool = (3,3)
act = 'relu'
dropout = 0.3
dense_layers = 1
units = 256
opt = 'rmsprop'
l = 'binary_crossentropy'
m = 'accuracy'
title = 'CNN Z-Scaled Data'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

cnn_results = []


model = build_cnn_model(title, layers, channels, k_size, pool, act, dropout, dense_layers, units)
model_cnn = seq_model_compile(title, model, opt, l, m)
history = cnn_model(title, model_cnn, X_train_scaled, X_test_scaled, y_train, y_test, epoch, batch, early_stop)
y_pred_probs = model_cnn.predict(X_test_scaled)
y_pred = (y_pred_probs > 0.5).astype(int).flatten()
cm = confusion_matrix(y_test, y_pred)
tn, fp, fn, tp = cm.ravel()
acc = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred, zero_division=0)
recall = recall_score(y_test, y_pred, zero_division=0)
f1 = f1_score(y_test, y_pred, zero_division=0)
cnn_results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'TP': tp,
        'TN': tn,
        'FP': fp,
        'FN': fn,
        'Accuracy': acc,
        'Precision': precision,
        'Recall': recall,
        'F1': f1
    })

cnn_results_df = pd.DataFrame(cnn_results)
display(cnn_results_df)
CNN Z-Scaled Data with 3 layer(s), 16 channels, (5, 5) kernel size, (3, 3) pool size, relu activation, 0.3 dropout, 1 hidden dense layer(s)
CNN Z-Scaled Data Compiled (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)
CNN Z-Scaled Data Tested (Optimizer: rmsprop, Loss: binary_crossentropy, Metric: accuracy)

CNN Z-Scaled Data (Batch Size=64, Epoch=100)
Epochs Trained: 19
Batch Size: 64
Final Training Accuracy: 0.989
Final Validation Accuracy: 0.965
Final Training Loss: 0.025
Final Validation Loss: 0.070
Test Accuracy: 0.978
Trainable Params: 1310209
No description has been provided for this image
8/8 ━━━━━━━━━━━━━━━━━━━━ 0s 36ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params TP TN FP FN Accuracy Precision Recall F1
0 CNN Z-Scaled Data 3 16 (5, 5) (3, 3) 0.3 1 256 64 19 1310209 207 18 4 1 0.978261 0.981043 0.995192 0.988067
In [115]:
# Lock Model
model_cnn.save('final_bird_model.keras')

Multi Class Prediction¶

In [140]:
def generate_multiclass_data(full_bird):
    X_list = []
    y_list = []

    for species in full_bird.keys():
        specs = full_bird[species]
        X_list.append(specs)
        y_list.extend([species] * specs.shape[0])

    X = np.concatenate(X_list, axis=0)
    y = np.array(y_list)
    X = X[..., np.newaxis]

    label = LabelEncoder()
    y_encoded = label.fit_transform(y)
    y_cat = to_categorical(y_encoded)


    X_flat = X.reshape(X.shape[0], -1)
    mean = X_flat.mean()
    std = X_flat.std()
    X = (X - mean) / std

    return train_test_split(X, y_cat, test_size=0.33, random_state=13, stratify=y_encoded), label

# Build CNN model
def build_multi_cnn_model(name, layers, channels, k_size, pool, act, drop, dense, u):
    print(f'{name} with {layers} layer(s), {channels} channels, {k_size} kernel size, {pool} pool, {act} activation, {drop} dropout, {dense} dense layer(s)')
    
    model = Sequential()
    model.add(Input(shape=(128, 517, 1)))
    model.add(Conv2D(filters=channels, kernel_size=k_size, padding='same', activation=act))
    model.add(MaxPooling2D(pool_size=pool))

    for i in range(1, layers):
        model.add(Conv2D(filters=channels * (2**i), kernel_size=k_size, padding='same', activation=act))
        model.add(MaxPooling2D(pool_size=pool))

    model.add(Flatten())
    model.add(Dropout(drop))

    units = u
    for j in range(dense):
        units = int(units)
        model.add(Dense(units=units, activation=act))
        if drop > 0:
            model.add(Dropout(drop)) 
        units = units/2  

    model.add(Dense(12, activation='softmax'))
    return model

# Compile Model
def seq_model_compile(title, model, opt, l, m):
    print(f'{title} (Optimizer: {opt}, Loss: {l}, Metric: {m})')
    model.compile(optimizer=opt, loss=l, metrics=[m])
    return model

# Train and evaluate
def cnn_model(title, model, xtrain, xtest, ytrain, ytest, e, b, stop):
    print(f'{title} (Batch Size={b}, Epoch={e})')
    history = model.fit(xtrain, ytrain, 
                        epochs=e, 
                        batch_size=b,
                        validation_data=(xtest, ytest),
                        callbacks=[stop],
                        verbose=0)
    plot_training_history(history, title)
    return history

def cnn_multi_metrics(model, X_test, y_test_cat, layers, channels, label_encoder):
    y_pred_probs = model.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test_cat, axis=1)

    acc = accuracy_score(y_true, y_pred)
    report = classification_report(y_true, y_pred, target_names=label_encoder.classes_, output_dict=True, zero_division=0)

    print(f"\nAccuracy: {acc:.3f}")
    print("\nClassification Report:")
    print(classification_report(y_true, y_pred, target_names=label_encoder.classes_, zero_division=0))

    
In [136]:
# Load and prepare data
(X_train, X_test, y_train, y_test), label_encoder = generate_multiclass_data(full_bird)

Number of layers optimizations¶

In [138]:
channels = 32
layers_list = [1,2,3]
k_size = (5, 5)
pool = (3, 3)
act = 'relu'
drop = 0.3
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for layers in layers_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)
CNN Multi-Class with 1 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 1s 38ms/step
CNN Multi-Class with 2 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 1s 65ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 79ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params Accuracy Macro F1 Macro Precision Macro Recall
0 CNN Multi-Class 1 32 (5, 5) (3, 3) 0.3 1 256 64 11 59183180 0.318043 0.040217 0.026504 0.083333
1 CNN Multi-Class 2 32 (5, 5) (3, 3) 0.3 1 256 64 11 13129868 0.467890 0.281314 0.358226 0.277578
2 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.3 1 256 64 10 2750732 0.521407 0.336088 0.412150 0.337448

Number of Dense Layers Optimization¶

In [142]:
channels = 32
layers = 3
k_size = (5, 5)
pool = (3, 3)
act = 'relu'
drop = 0.3
dense_layers_list = [1, 2]
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for dense_layers in dense_layers_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 78ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 2 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 79ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params Accuracy Macro F1 Macro Precision Macro Recall
0 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.3 1 256 64 13 2750732 0.524465 0.352835 0.415234 0.362651
1 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.3 2 256 64 13 2782092 0.529052 0.321278 0.344708 0.345752

Observations: 1 dense layer is enough for the model¶

Dropout Optimization¶

In [149]:
channels = 32
layers = 3
k_size = (5, 5)
pool = (3, 3)
act = 'relu'
drop_list = [0.2, 0.3, 0.4]
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for drop in drop_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': drop,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.2 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 82ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.3 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 80ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 80ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params Accuracy Macro F1 Macro Precision Macro Recall
0 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.2 1 256 64 13 2750732 0.495413 0.338714 0.368131 0.343732
1 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.3 1 256 64 12 2750732 0.498471 0.301184 0.326837 0.323927
2 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.4 1 256 64 15 2750732 0.524465 0.350077 0.398075 0.338919

Observations: Setting dropout to 0.4 improved accuracy, F1 and precision.¶

K Size Optimization¶

In [155]:
channels = 32
layers = 3
k_size_list = [(3, 3), (5, 5), (7, 7)]
pool = (3, 3)
act = 'relu'
drop = 0.4
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for k_size in k_size_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': dropout,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)
CNN Multi-Class with 3 layer(s), 32 channels, (3, 3) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 1s 48ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (5, 5) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 87ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 3s 154ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params Accuracy Macro F1 Macro Precision Macro Recall
0 CNN Multi-Class 3 32 (3, 3) (3, 3) 0.3 1 256 64 13 2586380 0.509174 0.347082 0.421334 0.351126
1 CNN Multi-Class 3 32 (5, 5) (3, 3) 0.3 1 256 64 14 2750732 0.521407 0.352820 0.410685 0.346536
2 CNN Multi-Class 3 32 (7, 7) (3, 3) 0.3 1 256 64 16 2997260 0.539755 0.386743 0.385072 0.399491

Observations: (7,7) kernel size performed the best with higher accuracy, F1 and Recall¶

Pool Optimization¶

In [159]:
channels = 32
layers = 3
k_size = (7, 7)
pool_list = [(2,2),(3, 3),(4,4)]
act = 'relu'
drop = 0.4
dense_layers = 1
units = 256
opt = 'rmsprop'
loss = 'categorical_crossentropy'
metric = 'accuracy'
title = 'CNN Multi-Class'
batch = 64
epoch = 100
early_stop = EarlyStopping(monitor='val_loss', patience=3, restore_best_weights=True)


results = []

for pool in pool_list:
    model = build_multi_cnn_model(title, layers, channels, k_size, pool, act, drop, dense_layers, units)
    model_cnn = seq_model_compile(title, model, opt, loss, metric)
    history = cnn_model(title, model_cnn, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

    y_pred_probs = model_cnn.predict(X_test)
    y_pred = np.argmax(y_pred_probs, axis=1)
    y_true = np.argmax(y_test, axis=1)

    acc = accuracy_score(y_true, y_pred)
    f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
    recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
    precision = precision_score(y_true, y_pred, average='macro', zero_division=0)

    results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': drop,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })

results_df = pd.DataFrame(results)
display(results_df)
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (2, 2) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 8s 355ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (3, 3) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 3s 135ms/step
CNN Multi-Class with 3 layer(s), 32 channels, (7, 7) kernel size, (4, 4) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 83ms/step
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params Accuracy Macro F1 Macro Precision Macro Recall
0 CNN Multi-Class 3 32 (7, 7) (2, 2) 0.4 1 256 64 11 34061324 0.467890 0.287067 0.316775 0.288464
1 CNN Multi-Class 3 32 (7, 7) (3, 3) 0.4 1 256 64 11 2997260 0.470948 0.246749 0.279161 0.254188
2 CNN Multi-Class 3 32 (7, 7) (4, 4) 0.4 1 256 64 13 1031180 0.490826 0.291996 0.413798 0.282692

Observations: Pooling (4,4) showed higher accuracy, F1, precission¶

Final model¶

In [169]:
# Final hyperparameters
layers = 3
channels = 32
k_size = (7, 7)
pool = (4, 4)
dropout = 0.4
dense_layers = 1
units = 256
batch = 64
epoch = 100
title = "CNN Multi-Class Final"
early_stop = EarlyStopping(monitor='val_loss', patience=5, restore_best_weights=True)

# Build and compile
model = build_multi_cnn_model(title, layers, channels, k_size, pool, 'relu', dropout, dense_layers, units)
model = seq_model_compile(title, model, 'rmsprop', 'categorical_crossentropy', 'accuracy')
history = cnn_model(title, model, X_train, X_test, y_train, y_test, epoch, batch, early_stop)

# Predict
y_pred_probs = model.predict(X_test)
y_pred = np.argmax(y_pred_probs, axis=1)
y_true = np.argmax(y_test, axis=1)

# Confusion matrix
cm = confusion_matrix(y_true, y_pred)
cm_df = pd.DataFrame(cm,
                     index=label_encoder.classes_,  
                     columns=label_encoder.classes_)  

print("Confusion Matrix (Table Format):")
display(cm_df)
CNN Multi-Class Final with 3 layer(s), 32 channels, (7, 7) kernel size, (4, 4) pool, relu activation, 0.4 dropout, 1 dense layer(s)
CNN Multi-Class Final (Optimizer: rmsprop, Loss: categorical_crossentropy, Metric: accuracy)
CNN Multi-Class Final (Batch Size=64, Epoch=100)
No description has been provided for this image
21/21 ━━━━━━━━━━━━━━━━━━━━ 2s 93ms/step
Confusion Matrix (Table Format):
amecro amerob bewwre bkcchi daejun houfin houspa norfli rewbla sonspa spotow whcspa
amecro 17 0 0 0 1 0 0 0 2 1 1 0
amerob 0 36 4 1 1 0 8 0 3 0 3 1
bewwre 0 0 21 0 2 3 5 0 3 10 2 1
bkcchi 0 1 1 0 1 0 2 0 2 3 1 4
daejun 0 0 8 0 25 0 2 0 1 2 2 1
houfin 1 3 1 0 1 8 11 0 0 2 1 0
houspa 1 3 4 0 4 0 189 0 2 1 3 1
norfli 1 4 0 0 1 1 2 0 0 1 2 0
rewbla 0 16 2 0 1 1 14 0 15 11 2 0
sonspa 2 10 10 0 3 2 8 0 3 41 5 3
spotow 2 3 10 0 0 0 5 0 3 2 20 0
whcspa 0 1 0 0 3 0 2 0 5 7 1 11
In [173]:
results = []
acc = accuracy_score(y_true, y_pred)
f1 = f1_score(y_true, y_pred, average='macro', zero_division=0)
recall = recall_score(y_true, y_pred, average='macro', zero_division=0)
precision = precision_score(y_true, y_pred, average='macro', zero_division=0)
results.append({
        'Title': title,
        'Layers': layers,
        'Channels': channels,
        'Kernel': k_size,
        'Pool': pool,
        'Dropout': drop,
        'Dense Layers': dense_layers,
        'Dense Units': units,
        'Batch Size': batch,
        'Epochs Trained': len(history.history['loss']),
        'Trainable Params': model.count_params(),
        'Accuracy': acc,
        'Macro F1': f1,
        'Macro Precision': precision,
        'Macro Recall': recall,
    })
results_df = pd.DataFrame(results)
display(results_df)
Title Layers Channels Kernel Pool Dropout Dense Layers Dense Units Batch Size Epochs Trained Trainable Params Accuracy Macro F1 Macro Precision Macro Recall
0 CNN Multi-Class Final <module 'tensorflow.keras.layers' from '/opt/a... 32 (7, 7) (4, 4) 0.4 1 256 64 22 1031180 0.585627 0.42703 0.437738 0.431629
In [175]:
model.save("cnn_bird_classifier.keras")

Generate Spectograms for test files¶

In [178]:
# Path
test_dir = '/Users/markdaza/Library/CloudStorage/OneDrive-SeattleUniversity/DATA 5322/Homework3/test_birds'
test_files = ['test1.mp3', 'test2.mp3', 'test3.mp3']

# Array for spectrograms
test_specs = np.zeros((128, 517, len(test_files)))

# Generate spectrograms
for i, fname in enumerate(test_files):
    full_path = os.path.join(test_dir, fname)
    y, sr = librosa.load(full_path, duration=3)
    
    # Compute mel spectrogram
    spec = librosa.feature.melspectrogram(y=y, sr=sr, hop_length=128, win_length=512, n_mels=128)
    spec_db = librosa.power_to_db(spec, ref=np.max)
    
    # Standardize w
    if spec_db.shape[1] < 517:
        pad_width = 517 - spec_db.shape[1]
        spec_db = np.pad(spec_db, ((0, 0), (0, pad_width)), mode='constant')
    else:
        spec_db = spec_db[:, :517]
    
    test_specs[:, :, i] = spec_db

print("All spectrograms processed. Shape:", test_specs.shape)
All spectrograms processed. Shape: (128, 517, 3)
In [180]:
plt.figure(figsize=(10, 4))
librosa.display.specshow(test_specs[:, :, 0], x_axis='time', y_axis='mel', sr=sr, cmap='gray_r')
plt.colorbar(format='%+2.0f dB')
plt.title("Mel Spectrogram: test1.mp3")
plt.tight_layout()
plt.show()
No description has been provided for this image
In [182]:
plt.figure(figsize=(10, 4))
librosa.display.specshow(test_specs[:, :, 1], x_axis='time', y_axis='mel', sr=sr, cmap='gray_r')
plt.colorbar(format='%+2.0f dB')
plt.title("Mel Spectrogram: test2.mp3")
plt.tight_layout()
plt.show()
No description has been provided for this image
In [184]:
plt.figure(figsize=(10, 4))
librosa.display.specshow(test_specs[:, :, 2], x_axis='time', y_axis='mel', sr=sr, cmap='gray_r')
plt.colorbar(format='%+2.0f dB')
plt.title("Mel Spectrogram: test3.mp3")
plt.tight_layout()
plt.show()
No description has been provided for this image
In [194]:
# Extract mean and std from training set
mean = X_train.mean()
std = X_train.std()

# Apply Z-scaling to spectrograms
test_specs_scaled = (test_specs - mean) / std
test_specs_scaled = np.transpose(test_specs_scaled, (2, 0, 1)) 
test_specs_scaled = test_specs_scaled[..., np.newaxis]  

# Load Model
model_multi = load_model('cnn_bird_classifier.keras')

# Predict 
y_pred_probs = model_multi.predict(test_specs_scaled)  

# Get predictions for each spectrogram
top3_indices = np.argsort(y_pred_probs, axis=1)[:, -3:][:, ::-1] 
top3_probs = np.sort(y_pred_probs, axis=1)[:, -3:][:, ::-1]

# Decode labels of predictions to bird species
top3_labels = label_encoder.inverse_transform(top3_indices.flatten()).reshape(3, 3)

# Top predictions
for i, (labels, probs) in enumerate(zip(top3_labels, top3_probs)):
    print(f"\ntest{i+1}.mp3 - Top 3 Predictions:")
    for j in range(3):
        print(f"  {j+1}. {labels[j]} (prob: {probs[j]:.3f})")
1/1 ━━━━━━━━━━━━━━━━━━━━ 0s 37ms/step

test1.mp3 - Top 3 Predictions:
  1. houspa (prob: 1.000)
  2. rewbla (prob: 0.000)
  3. whcspa (prob: 0.000)

test2.mp3 - Top 3 Predictions:
  1. houspa (prob: 0.904)
  2. spotow (prob: 0.096)
  3. rewbla (prob: 0.000)

test3.mp3 - Top 3 Predictions:
  1. houspa (prob: 1.000)
  2. spotow (prob: 0.000)
  3. whcspa (prob: 0.000)